# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import codecs
import json

from scrapy.pipelines.images import ImagesPipeline
from scrapy.exporters import JsonItemExporter
import pymysql
from pymysql.cursors import DictCursor
from twisted.enterprise import adbapi


class CnblogsscrapyPipeline:
    def process_item(self, item, spider):
        return item


class CnblogsImagesPipeline(ImagesPipeline):
    def item_completed(self, results, item, info):
        if "front_image_url" in item:
            image_file_path = ""
            try:
                for ok, x in results:
                    image_file_path = x['path']
            except Exception as e:
                print(f'图片下载错误： {e}')
            item["front_image_path"] = image_file_path
        return item


class JsonWithEncodingPipeline:
    """
    自定义json文件的导出
    """
    def __init__(self):
        self.file = codecs.open('data.json', 'a', encoding='utf-8')

    def process_item(self, item, spider):
        lines = json.dumps(dict(item), ensure_ascii=False) + '\n'
        self.file.write(lines)
        return item

    def spider_closed(self, spider):
        self.file.close()


class JsonExporterPipeline:
    """
    scrapy自己的exporter，这两种方法任意选择
    """
    def __init__(self):
        self.file = open('data_2.json', 'wb')
        self.exporter = JsonItemExporter(self.file, encoding='utf-8', ensure_ascii=False)
        self.exporter.start_exporting()

    def process_item(self, item, spider):
        self.exporter.export_item(item)
        return item

    def spider_closed(self, spider):
        self.exporter.finish_exporting()
        self.file.close()


class MysqlTwistedPipline:
    def __init__(self, dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls, settings):
        dbparms = dict(
            host=settings["MYSQL_HOST"],
            db=settings["MYSQL_DATABASE"],
            user=settings["MYSQL_USER"],
            passwd=settings["MYSQL_PASSWORD"],
            charset='utf8',
            cursorclass=pymysql.cursors.DictCursor,
            use_unicode=True
        )
        dbpool = adbapi.ConnectionPool("pymysql", **dbparms)
        return cls(dbpool)

    def process_item(self, item, spider):
        # 使用twisted将mysql插入变成异步执行
        query = self.dbpool.runInteraction(self.do_insert, item)
        query.addErrback(self.handle_error, item, spider)  # 处理异常
        return item

    def handle_error(self, failure, item, spider):
        # 处理异步插入的异常
        print(failure)

    def do_insert(self, cursor, item):
        # 执行具体的插入
        # 根据不同的item 构建不同的sql语句并插入到mysql中
        print("开始执行插入")
        print(item.get_insert_sql())
        insert_sql, params = item.get_insert_sql()

        cursor.execute(insert_sql, params)


class MysqlPipeline:
    """
    同步写法，不建议，因为会很容易阻塞住，建议和异步框架保持一致，同样使用异步IO，
    """
    def __init__(self):
        self.connection = pymysql.connect(host='localhost',
                                          port=3306,
                                          user='root',
                                          password='123456',
                                          db='cnblogs',
                                          charset='utf8')
        self.cursor = self.connection.cursor()

    def process_item(self, item, spider):
        insert_sql = """
            insert into `cn_news` (`url`, `id`, `title`, `create_time`, 
            `content`, `tags`, `front_image_url`, `front_image_path`, 
            `commentCount`, `diggCount`, `totalView`, `md5_id`)
            values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
            on duplicate key update `title`=values(title), `content`=values(content), commentCount=values(commentCount),
            diggCount=values(diggCount), `totalView`=values(totalView)
        """  # 最后两行是当主键冲突的时候强制更新
        params = list()
        params.append(item.get('url', ""))
        params.append(item.get('id', 0))
        params.append(item.get('title', ""))
        params.append(item.get('create_time', "1111-1-1"))
        params.append(item.get('content', ""))
        params.append(item.get('tags', ""))
        front_image = ",".join(item.get('front_image_url', []))
        params.append(front_image)
        params.append(item.get('front_image_path', ""))
        params.append(item.get('commentCount', 0))
        params.append(item.get('diggCount', 0))
        params.append(item.get('totalView', 0))
        params.append(item.get('md5_id', ""))
        try:
            self.cursor.execute(insert_sql, tuple(params))
            self.connection.commit()
        except Exception as e:
            print(f"插入数据有错误，错误为：{e}")
        finally:
            self.connection.close()
        return item